# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate: 
# Message:  获取主组及分组表,根据车型下一级数量来定

from MysqlTools import MysqlTools
import requests
import json
import os
import hashlib
import time
import sys
import redis


class GetGroupStrctureTable:

    def __init__(self, **kwargs):
        self.db = MysqlTools(db_config=kwargs.get("db_config"), type='dict', auto_commit=1000)
        self.db.num = 648000
        self.base_path = '/Users/duanmengdi/Desktop/workspace/py3_code/JAC_html/gs_html/'  # 文件保存的路径
        self.img_path = '/Users/duanmengdi/Desktop/workspace/py3_code/JAC_img/'
        self.insert_table1 = kwargs.get("insert_table1")
        self.insert_table2 = kwargs.get("insert_table2")
        self.cid_table = kwargs.get("cid_table")
        self.r = redis.Redis(host='localhost', port=6379, db=0).pipeline()  # redis链接,用于断点续爬
        self.redis_set = 'JAC_set_new'

        self.base_url = 'http://218.104.65.183:8080/Ext/data/SpareInfo.ashx'
        self.cookies = {
            "ASP.NET_SessionId": "lxayqhhskmsgh51be30j04zj",
            "languageS": "China",
            "Userlanguages": "China",
            "security_session_verify": "28d46281aacbc1fed03397f7c3dcb063",
            # "": "",
        }
        self.headers = {
            "Origin": "http://218.104.65.183:8080",
            "X-Requested-With": "XMLHttpRequest",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
            "Referer": "http://218.104.65.183:8080/Ext/MainPage.aspx",
            # "": "",
            # "": "",
        }
        self.LoadTreeChildNodes_form = {
            "type": "",
            "nodeid": "",
            "vincode": "0",
            "factory": "",
            "rootoid": "-1",
        }

    def get_md5(self, str):
        '''获取md5加密电子指纹'''
        md5 = hashlib.md5()
        md5.update(str)
        return md5.hexdigest()

    def get_dc(self):
        _dc = str(time.time()).replace('.', '')[:13]
        return _dc

    def parse_url(self, url, method, post_data=None, headers=None, tofile=True, **kwargs):
        requests.packages.urllib3.disable_warnings()  # 因为把ssl安全验证关了,所以不让代码报不安全的https请求警告
        key = kwargs.get("key", "")
        charset = kwargs.get("charset", "utf8")  # 暂时没有用到
        allow_redirects = kwargs.get("allow_redirects", False)  # 是否允许重定向, 默认不重定向,获取原始响应
        if method == 'GET':
            data = "GET" + url + key  # 拼接data
            md5 = self.get_md5(data)  # 获取此次请求电子指纹
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                # print("GET for local")
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:  # 如果路径不存在, 从web端请求
                print("GET for web")
                try:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.cookies, timeout=30,
                                            allow_redirects=allow_redirects)
                except Exception as e:
                    response = requests.get(url=url, verify=False, headers=headers, cookies=self.cookies, timeout=30,
                                            allow_redirects=allow_redirects)

                # 每次请求都获取cookies, 并更新self.cookies(如果获取到cookies的话)
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.cookies[k] = cookies[k]

                # 获取location,
                location = response.headers.get("Location", None)
                if location:
                    print("响应头中存在location")
                    self.location = location
                    print(location)
                    # return response.content.decode('utf8')

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                else:
                    response = None

        elif method == "POST":
            data = "POST" + url + json.dumps(post_data)
            md5 = self.get_md5(data.encode())
            file_path = self.base_path + md5 + ".html"
            if os.path.exists(file_path):  # 判断文件是否存在,如果存在就从本地读取
                # print("POST for local")
                with open(file_path, 'r') as f:
                    res = f.read()
                response = res
            else:
                print("POST for web")
                try:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers,
                                             cookies=self.cookies, timeout=30,
                                             allow_redirects=allow_redirects)
                except Exception as e:
                    response = requests.post(url=url, data=post_data, verify=False, headers=headers,
                                             cookies=self.cookies, timeout=30,
                                             allow_redirects=allow_redirects)

                # 获取cookies, 并更新self.cookies
                cookies = response.cookies.get_dict()
                for k in cookies:
                    self.cookies[k] = cookies[k]

                # 获取location,
                location = response.headers.get("Location", None)
                if location:
                    print("响应头中存在location")
                    self.location = location
                    print(location)
                    # response, file_path = self.parse_url(url=location, method="GET", headers=headers, local_pid=key)
                    # print "已重新请求location地址"
                    # return response, file_path
                    # return response.content.decode('utf8')

                if response:
                    if tofile:
                        with open(file_path, 'w') as f:  # 保存此次请求的html
                            f.write(response.content.decode('utf8'))
                    response = response.content.decode("utf8")
                else:
                    response = None
        return response, file_path

    def get_cid_list(self):
        sql = '''select distinct `code` from `{}`'''.format(self.cid_table)
        data = self.db.select_all(sql)
        _l = [i.get("code") for i in data]
        return _l
        # return ['4878689_2020','4878024_2020','4511788_2020']
        # return ['3688061_2010']

    def get_hotcoords(self, img_oid):
        GetHotCoords_form = {
            "pid": img_oid,
            "type": "GetHotCoords"
        }
        try:
            response, file_path = self.parse_url(url=self.base_url, method="POST", post_data=GetHotCoords_form,
                                                 headers=self.headers)
        except:
            response, file_path = self.parse_url(url=self.base_url, method="POST", post_data=GetHotCoords_form,
                                                 headers=self.headers)
        if not response:
            return {}
        info_l = response.split("@")
        _dict = {i.split("#")[2]: {"x": i.split("#")[0], "y": i.split("#")[1]} for i in info_l}
        return _dict

    def save_img(self, cid, spareoid, mid, img_url):
        img_path = self.img_path + cid + '/'
        if not os.path.exists(img_path):
            os.mkdir(img_path)
        img_name = mid + '.png'
        if os.path.exists(img_path + img_name):
            return
        try:
            response = requests.get(url=img_url, headers=self.headers, cookies=self.cookies, timeout=40)
        except:
            try:
                response = requests.get(url=img_url, headers=self.headers, cookies=self.cookies, timeout=40)
            except:
                with open('./img_error.txt', 'a') as i:
                    i.write('{}_{}_{}'.format(cid, mid, img_url))
                    i.write('\n')
                    return

        with open(img_path + img_name, 'wb') as f:
            f.write(response.content)

    def get_img_hotcoords(self, cid, spareoid, mid, factory):
        SparePicCount_form = {
            "spareoid": spareoid,
            "factory": factory,
            'type': "SparePicCount"
        }
        try:
            response, file_path = self.parse_url(url=self.base_url, method='POST', post_data=SparePicCount_form,
                                                 headers=self.headers)
        except:
            response, file_path = self.parse_url(url=self.base_url, method='POST', post_data=SparePicCount_form,
                                                 headers=self.headers)
        if response == 'fail' or response == '':
            img_oid = '0'
            img_path = ''
            hotcoords_dict = self.get_hotcoords(img_oid)
            return hotcoords_dict
        info_l = response.split("#")
        img_oid = info_l[0]
        img_path = info_l[1]
        img_url = 'http://218.104.65.183:8080/Ext/' + img_path.replace('../Ext/', '')
        hotcoords_dict = self.get_hotcoords(img_oid)
        self.save_img(cid, spareoid, mid, img_url)
        return hotcoords_dict  # 可以在此处直接把图片描点请求下来

    def LoadTreeChildNodes(self, nodeid, factory):
        self.LoadTreeChildNodes_form["type"] = "LoadTreeChildNodes"
        self.LoadTreeChildNodes_form["nodeid"] = nodeid
        self.LoadTreeChildNodes_form["factory"] = factory
        try:
            response, file_path = self.parse_url(url=self.base_url, method="POST",
                                                 post_data=self.LoadTreeChildNodes_form, headers=self.headers)
        except:
            response, file_path = self.parse_url(url=self.base_url, method="POST",
                                                 post_data=self.LoadTreeChildNodes_form, headers=self.headers)
        l1 = response.split("$")
        if not response:
            return []
        l2 = []
        print(file_path)
        for i in l1:
            l = i.split('@')
            _dict = {
                'nodeid': l[0],
                'name': l[1],
                'img': l[2],
                'partnumber': l[3],
                'isspare': l[4],
                's_time': l[5],
                'e_time': l[6],
                'psort': l[7],
                'qty': l[8],
                'remark': l[9],
                'factory': l[10],
                '11': l[11],
                '12': l[12],
                'isstop': l[13],
                '14': l[14],
                '15': l[15],
                'nextcount': l[16],
                'othername': l[17],
                '18': l[18],
                '19': l[19],
                '20': l[20]
            }
            l2.append(_dict)
        # l2 = [i.split("@") for i in l1]
        return l2

    def save2db(self, level1, level1_code, level2, level2_code, level3, level3_code, level4, level4_code, level5,
                level5_code, level6, level6_code, level7, level7_code, imgPoint, raw_info):
        _dict = dict(
            level1=level1,
            level1_code=level1_code,
            level2=level2,
            level2_code=level2_code,
            level3=level3,
            level3_code=level3_code,
            level4=level4,
            level4_code=level4_code,
            level5=level5,
            level5_code=level5_code,
            level6=level6,
            level6_code=level6_code,
            level7=level7,
            level7_code=level7_code,
            imgPoint=imgPoint,
            raw_info=raw_info,
        )
        in_sql = self.db.get_insert_sql('all_level_info_G', _dict)
        self.db.execute_sql(in_sql)

    def handle_group(self, cid, list):
        # 主组处理函数
        pass

    def handle_structrue(self, cid, list):
        # 分组处理函数
        level1_nodeid = cid.split('_')[0]
        factory = cid.split('_')[1]
        num = 0
        for level3_info in list:
            num += 1
            # 最终删除掉本层,让第四层级直接对应主组层
            level3_nodeid = level3_info.get("nodeid")
            level3_name = level3_info.get("name")
            level3_number = level3_info.get("partnumber")
            ### 请求第三层的图片及描点
            level3_hotcoords_dict = self.get_img_hotcoords(cid, level3_nodeid, level3_number, factory)

            level4_l = self.LoadTreeChildNodes(level3_nodeid, factory)
            count = len(level4_l)
            for level4_info in level4_l:
                # 第四层级,已经是我们的分组层级,要去请求图片及描点信息,构建描点信息字典,供第五层级提取
                level4_nodeid = level4_info.get("nodeid")
                level4_name = level4_info.get("name")
                level4_number = level4_info.get("partnumber")
                level4_psort = level4_info.get("psort")
                level4_nextcount = level4_info.get("nextcount")
                level4_imgPoint = level3_hotcoords_dict.get(level4_psort, {})
                if level4_nextcount == "0":
                    raw_info = {
                        "level1": {},
                        "level2": {},
                        "level3": level3_info,
                        "level4": level4_info,
                        "level5": {},
                        "level6": {},
                        "level7": {},
                        'imgPoint_dict': level3_hotcoords_dict
                        # "level5":'',
                    }
                    self.save2db(
                        level1='',
                        level1_code=cid,
                        level2='',
                        level2_code='',
                        level3=level3_name,
                        level3_code=level3_number,
                        level4=level4_name,
                        level4_code=level4_number,
                        level5='',
                        level5_code='',
                        level6='',
                        level6_code='',
                        level7='',
                        level7_code='',
                        imgPoint=json.dumps(level4_imgPoint),
                        raw_info=json.dumps(raw_info),
                    )
                else:
                    level5_l = self.LoadTreeChildNodes(level4_nodeid, factory)
                    count = len(level5_l)
                    if count == 0:
                        raw_info = {
                            "level1": {},
                            "level2": {},
                            "level3": level3_info,
                            "level4": level4_info,
                            "level5": {},
                            "level6": {},
                            "level7": {},
                            'imgPoint_dict': level3_hotcoords_dict
                            # "level5":'',
                        }
                        self.save2db(
                            level1='',
                            level1_code=cid,
                            level2='',
                            level2_code='',
                            level3=level3_name,
                            level3_code=level3_number,
                            level4=level4_name,
                            level4_code=level4_number,
                            level5='',
                            level5_code='',
                            level6='',
                            level6_code='',
                            level7='',
                            level7_code='',
                            imgPoint=json.dumps(level4_imgPoint),
                            raw_info=json.dumps(raw_info),
                        )
                    else:
                        hotcoodrs_dict = self.get_img_hotcoords(cid, level4_nodeid, level4_number,
                                                                factory)  # 获取描点字典,保存图片
                        for level5_info in level5_l:
                            # 第五层级,已经是我们的零件号层级,但是还会有部分零件号依然存在下一层级即第六层级
                            level5_nodeid = level5_info.get("nodeid")
                            level5_name = level5_info.get("name")
                            level5_number = level5_info.get("partnumber")
                            level5_nextcount = level5_info.get("nextcount")
                            level5_psort = level5_info.get("psort")
                            level5_imgPoint = hotcoodrs_dict.get(level5_psort, {})
                            if level5_nextcount == "0":
                                raw_info = {
                                    "level1": {},
                                    "level2": {},
                                    "level3": level3_info,
                                    "level4": level4_info,
                                    'level5': level5_info,
                                    'level6': {},
                                    'level7': {},
                                    'imgPoint_dict': hotcoodrs_dict
                                }
                                self.save2db(
                                    level1='',
                                    level1_code=cid,
                                    level2='',
                                    level2_code='',
                                    level3=level3_name,
                                    level3_code=level3_number,
                                    level4=level4_name,
                                    level4_code=level4_number,
                                    level5=level5_name,
                                    level5_code=level5_number,
                                    level6='',
                                    level6_code='',
                                    level7='',
                                    level7_code='',
                                    imgPoint=json.dumps(level5_imgPoint),
                                    raw_info=json.dumps(raw_info),
                                )
                            else:
                                level6_l = self.LoadTreeChildNodes(level5_nodeid, factory)
                                count = len(level6_l)
                                if count == 0:
                                    raw_info = {
                                        "level1": {},
                                        "level2": {},
                                        "level3": level3_info,
                                        "level4": level4_info,
                                        'level5': level5_info,
                                        'level6': {},
                                        'level7': {},
                                        'imgPoint_dict': hotcoodrs_dict
                                    }
                                    self.save2db(
                                        level1='',
                                        level1_code=cid,
                                        level2='',
                                        level2_code='',
                                        level3=level3_name,
                                        level3_code=level3_number,
                                        level4=level4_name,
                                        level4_code=level4_number,
                                        level5=level5_name,
                                        level5_code=level5_number,
                                        level6='',
                                        level6_code='',
                                        level7='',
                                        level7_code='',
                                        imgPoint=json.dumps(level5_imgPoint),
                                        raw_info=json.dumps(raw_info),
                                    )
                                for level6_info in level6_l:
                                    level6_name = level6_info.get("name")
                                    level6_number = level6_info.get("partnumber")
                                    raw_info = {
                                        "level1": {},
                                        "level2": {},
                                        "level3": level3_info,
                                        "level4": level4_info,
                                        'level5': level5_info,
                                        'level6': level6_info,
                                        'level7': {},
                                        'imgPoint_dict': hotcoodrs_dict
                                    }
                                    self.save2db(
                                        level1='',
                                        level1_code=cid,
                                        level2='',
                                        level2_code='',
                                        level3=level3_name,
                                        level3_code=level3_number,
                                        level4=level4_name,
                                        level4_code=level4_number,
                                        level5=level5_name,
                                        level5_code=level5_number,
                                        level6=level6_name,
                                        level6_code=level6_number,
                                        level7='',
                                        level7_code='',
                                        imgPoint=json.dumps(level5_imgPoint),
                                        raw_info=json.dumps(raw_info),
                                    )
            # if num == 3:
            #     self.db.commit_sql()
            #     exit(0)
        self.db.commit_sql()

    def run(self):
        cid_list = self.get_cid_list()
        for cid in cid_list:
            # time.sleep(1)
            print(cid)
            # self.r.sismember(self.redis_set, cid)
            # if self.r.execute()[0]:
            #     print(cid, "跳过")
            #     continue
            # self.r.sadd(self.redis_set, cid)
            nodeid = cid.split('_')[0]
            factory = cid.split('_')[1]
            res_l = self.LoadTreeChildNodes(nodeid, factory)
            count = len(res_l)
            if count < 25:  # 目前界限为22
                # 进入主组处理函数
                self.handle_structrue(cid, res_l)
                self.r.execute()

            else:
                # 进入分组处理函数
                # self.handle_structrue(cid, res_l)
                # self.r.execute()
                pass



if __name__ == '__main__':
    db_config = dict(
        host="127.0.0.1",
        port=3306,
        user="root",
        passwd="123456",
        db="JAC",
        charset="utf8",
    )
    cid_table = 'cars_info_collection'
    insert_table1 = 'strcture_name'
    insert_table2 = 'cars_info_collection'
    tools = GetGroupStrctureTable(db_config=db_config, insert_table1=insert_table1, insert_table2=insert_table2,
                                  cid_table=cid_table)
    tools.run()